{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tqdm\n",
      "  Downloading http://repo.myhuaweicloud.com/repository/pypi/packages/f3/76/4697ce203a3d42b2ead61127b35e5fcc26bba9a35c03b32a2bd342a4c869/tqdm-4.46.1-py2.py3-none-any.whl (63kB)\n",
      "\u001b[K    100% |████████████████████████████████| 71kB 31.6MB/s ta 0:00:01\n",
      "\u001b[?25hInstalling collected packages: tqdm\n",
      "Successfully installed tqdm-4.46.1\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 20.1.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Collecting lightgbm\n",
      "  Downloading http://repo.myhuaweicloud.com/repository/pypi/packages/0b/9d/ddcb2f43aca194987f1a99e27edf41cf9bc39ea750c3371c2a62698c509a/lightgbm-2.3.1-py2.py3-none-manylinux1_x86_64.whl (1.2MB)\n",
      "\u001b[K    100% |████████████████████████████████| 1.2MB 66.1MB/s ta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: scikit-learn in /home/ma-user/anaconda3/envs/XGBoost-Sklearn/lib/python3.6/site-packages (from lightgbm)\n",
      "Requirement already satisfied: scipy in /home/ma-user/anaconda3/envs/XGBoost-Sklearn/lib/python3.6/site-packages (from lightgbm)\n",
      "Requirement already satisfied: numpy in /home/ma-user/anaconda3/envs/XGBoost-Sklearn/lib/python3.6/site-packages (from lightgbm)\n",
      "Installing collected packages: lightgbm\n",
      "Successfully installed lightgbm-2.3.1\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 20.1.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "! pip install tqdm\n",
    "! pip install lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Using MoXing-v1.15.1-99273b13\n",
      "INFO:root:Using OBS-Python-SDK-3.1.2\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import mean_squared_error,explained_variance_score\n",
    "from sklearn.model_selection import KFold\n",
    "import lightgbm as lgb\n",
    "import math\n",
    "test_data_path = './data/A_testData0531.csv'\n",
    "train_gps_path = './data/train0523.csv'\n",
    "port_path = './data/port.csv'\n",
    "result_path = './result/result_server_20200616.csv'\n",
    "\n",
    "import moxing as mox\n",
    "OBS_DATA_PATH = \"s3://ship-eta/data/train0523.csv\"\n",
    "OBS_PORT_PATH = \"s3://ship-eta/data/port.csv\"\n",
    "OBS_TEST_PATH = \"s3://ship-eta/data/A_testData0531.csv\"\n",
    "OBS_RES_PATH =  \"s3://ship-eta/result/result_server_20200616.csv\"\n",
    "mox.file.copy_parallel(OBS_DATA_PATH, train_gps_path)\n",
    "mox.file.copy_parallel(OBS_PORT_PATH, port_path)\n",
    "mox.file.copy_parallel(OBS_TEST_PATH, test_data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_data_type(data, mode='train'):\n",
    "    if mode=='test':\n",
    "        data['onboardDate'] = pd.to_datetime(data['onboardDate'], infer_datetime_format=True)\n",
    "        data['temp_timestamp'] = data['timestamp']\n",
    "        data['ETA'] = None\n",
    "        data['creatDate'] = None\n",
    "    data['loadingOrder'] = data['loadingOrder'].astype(str)\n",
    "    data['timestamp'] = pd.to_datetime(data['timestamp'], infer_datetime_format=True)\n",
    "    data['longitude'] = data['longitude'].astype(float)\n",
    "    data['latitude'] = data['latitude'].astype(float)\n",
    "    data['speed'] = data['speed'].astype(float)\n",
    "    data['TRANSPORT_TRACE'] = data['TRANSPORT_TRACE'].astype(str)\n",
    "    return data\n",
    "\n",
    "def get_test_data_info(path):\n",
    "    data = pd.read_csv(path) \n",
    "    test_trace_set = data['TRANSPORT_TRACE'].unique()\n",
    "    test_order_belong_to_trace = {}\n",
    "    for item in test_trace_set:\n",
    "        orders = data[data['TRANSPORT_TRACE'] == item]['loadingOrder'].unique()\n",
    "        test_order_belong_to_trace[item] = orders\n",
    "    return format_data_type(data, mode='test'), test_trace_set, test_order_belong_to_trace\n",
    "\n",
    "test_data_origin, test_trace_set, test_order_belong_to_trace = get_test_data_info(test_data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_port_info():\n",
    "    port_data = {}\n",
    "    test_port_set = set()\n",
    "    for route in test_trace_set:\n",
    "        ports = route.split('-')\n",
    "        test_port_set = set.union(test_port_set, set(ports))\n",
    "    port_data_origin = pd.read_csv(port_path)\n",
    "    for item in port_data_origin.itertuples():\n",
    "        if getattr(item, 'TRANS_NODE_NAME') in test_port_set:\n",
    "            port_data[getattr(item, 'TRANS_NODE_NAME')] = {'LONGITUDE': getattr(item, 'LONGITUDE'),'LATITUDE': getattr(item, 'LATITUDE') }\n",
    "    return port_data\n",
    "port_data = get_port_info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 22/22 [30:37<00:00, 83.54s/it]\n"
     ]
    }
   ],
   "source": [
    "def get_train_order_data():\n",
    "    train_data = {}\n",
    "    train_data_origin = pd.read_csv(train_gps_path, usecols = [0,2,3,4,6,12], header=None)\n",
    "    for route in tqdm(test_trace_set):\n",
    "        ports = route.split(\"-\")\n",
    "        start_port = ports[0]\n",
    "        dest_port = ports[-1]\n",
    "        valid_order_name = train_data_origin[train_data_origin[12].apply(lambda x: str(x).startswith(start_port) and (dest_port in str(x)))][0].unique()\n",
    "        valid_order_info = train_data_origin[train_data_origin[0].isin(valid_order_name)]\n",
    "        valid_order_info.columns = ['loadingOrder','timestamp','longitude','latitude','speed', 'TRANSPORT_TRACE']\n",
    "        # valid_order_info['timestamp'] = pd.to_datetime(valid_order_info['timestamp'], infer_datetime_format=True)\n",
    "        train_data[route] = valid_order_info\n",
    "        train_data[route]['timestamp'] = pd.to_datetime(train_data[route]['timestamp'], infer_datetime_format=True)\n",
    "    return train_data\n",
    "\n",
    "train_order_data = get_train_order_data()\n",
    "\n",
    "for item in train_order_data:\n",
    "    if (train_order_data[item].shape[0] == 0):\n",
    "        print(\"error == \", item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_data(route_order_info, route):\n",
    "    ports = route.split(\"-\")\n",
    "    dest_port = ports[-1]\n",
    "    dest_longitude = port_data[dest_port]['LONGITUDE']\n",
    "    dest_latitude = port_data[dest_port]['LATITUDE']\n",
    "    train_data = None\n",
    "    order_list = route_order_info['loadingOrder'].unique()\n",
    "    for order in order_list:\n",
    "        order_info_set = route_order_info[route_order_info['loadingOrder'] == order].sort_values(by='timestamp')\n",
    "#         order_info_set['timestamp'] = pd.to_datetime(order_info_set['timestamp'], infer_datetime_format=True)\n",
    "#       获取起航时间\n",
    "        for info_item in order_info_set.itertuples():\n",
    "            if getattr(info_item, 'speed') > 0:\n",
    "                start_time = getattr(info_item, 'timestamp')\n",
    "                break\n",
    "#       获取到达目的地时间，这里需要用 GPS 判断\n",
    "        end_time = order_info_set['timestamp'].max()\n",
    "        for info_item in order_info_set.itertuples():\n",
    "            if abs(getattr(info_item, 'longitude') - dest_longitude) < 0.3 and abs(getattr(info_item, 'latitude') - dest_latitude) < 0.3:\n",
    "                end_time = min(end_time, getattr(info_item, 'timestamp'))\n",
    "                break\n",
    "#         人工截取前 40% 的数据   \n",
    "        cut_size = math.ceil(order_info_set.shape[0]*0.4)\n",
    "        order_info_set = order_info_set[0:cut_size]\n",
    "        \n",
    "#         获取经纬度速度信息\n",
    "        agg_function = ['min', 'max', 'mean', 'median']\n",
    "        agg_col = ['latitude', 'longitude', 'speed']\n",
    "        feature_temp = order_info_set.groupby('loadingOrder')[agg_col].agg(agg_function).reset_index()\n",
    "        feature_temp.columns = ['loadingOrder'] + ['{}_{}'.format(i, j) for i in agg_col for j in agg_function]\n",
    "#         算出航行用时\n",
    "        feature_temp['label'] = (end_time - start_time).total_seconds()\n",
    "        train_data = pd.concat([train_data,feature_temp])\n",
    "#     人工补足训练数据\n",
    "    if (train_data.shape[0] < 10):\n",
    "        for i in range(5):\n",
    "            train_data = pd.concat([train_data,train_data])\n",
    "    \n",
    "    return train_data.reset_index(drop=True)  \n",
    "\n",
    "def get_test_data(order):\n",
    "    order_info_set = test_data_origin[test_data_origin['loadingOrder'] == order].sort_values(by='timestamp')\n",
    "    agg_function = ['min', 'max', 'mean', 'median']\n",
    "    agg_col = ['latitude', 'longitude', 'speed']\n",
    "    feature = order_info_set.groupby('loadingOrder')[agg_col].agg(agg_function).reset_index()\n",
    "    feature.columns = ['loadingOrder'] + ['{}_{}'.format(i, j) for i in agg_col for j in agg_function]\n",
    "    return feature.reset_index(drop=True)\n",
    "def mse_score_eval(preds, valid):\n",
    "    labels = valid.get_label()\n",
    "    scores = mean_squared_error(y_true=labels, y_pred=preds)\n",
    "    return 'mse_score', scores, True\n",
    "def train_model(x, y, seed=981125, is_shuffle=True):\n",
    "    train_pred = np.zeros((x.shape[0], ))\n",
    "    n_splits = min(5, x.shape[0])\n",
    "    # Kfold\n",
    "    fold = KFold(n_splits=n_splits, shuffle=is_shuffle, random_state=seed)\n",
    "    kf_way = fold.split(x)\n",
    "    # params\n",
    "    params = {\n",
    "        'learning_rate': 0.01,\n",
    "        'boosting_type': 'gbdt',\n",
    "        'objective': 'regression',\n",
    "        'num_leaves': 36,\n",
    "        'feature_fraction': 0.6,\n",
    "        'bagging_fraction': 0.7,\n",
    "        'bagging_freq': 6,\n",
    "        'seed': 8,\n",
    "        'bagging_seed': 1,\n",
    "        'feature_fraction_seed': 7,\n",
    "        'min_data_in_leaf': 25,\n",
    "        'nthread': 8,\n",
    "        'verbose': 1,\n",
    "    }\n",
    "    # train\n",
    "    for n_fold, (train_idx, valid_idx) in enumerate(kf_way, start=1):\n",
    "        train_x, train_y = x.iloc[train_idx], y.iloc[train_idx]\n",
    "        valid_x, valid_y = x.iloc[valid_idx], y.iloc[valid_idx]\n",
    "        # 数据加载\n",
    "        n_train = lgb.Dataset(train_x, label=train_y)\n",
    "        n_valid = lgb.Dataset(valid_x, label=valid_y)\n",
    "        clf = lgb.train(\n",
    "            params=params,\n",
    "            train_set=n_train,\n",
    "            num_boost_round=3000,\n",
    "            valid_sets=[n_valid],\n",
    "            early_stopping_rounds=100,\n",
    "            verbose_eval=100,\n",
    "            feval=mse_score_eval\n",
    "        )\n",
    "        train_pred[valid_idx] = clf.predict(valid_x, num_iteration=clf.best_iteration)\n",
    "    return clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/22 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.76053e+11\tvalid_0's mse_score: 3.76053e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.54089e+11\tvalid_0's mse_score: 4.54089e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 5.69511e+10\tvalid_0's mse_score: 5.69511e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 8.68981e+10\tvalid_0's mse_score: 8.68981e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.2237e+11\tvalid_0's mse_score: 1.2237e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.68296e+11\tvalid_0's mse_score: 1.68296e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 2.08223e+11\tvalid_0's mse_score: 2.08223e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.45238e+11\tvalid_0's mse_score: 2.45238e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.89532e+11\tvalid_0's mse_score: 1.89532e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.47304e+11\tvalid_0's mse_score: 2.47304e+11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▍         | 1/22 [00:35<12:25, 35.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.92915e+11\tvalid_0's mse_score: 3.92915e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 6.4702e+11\tvalid_0's mse_score: 6.4702e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 4.25867e+11\tvalid_0's mse_score: 4.25867e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 6.86659e+11\tvalid_0's mse_score: 6.86659e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.68181e+11\tvalid_0's mse_score: 3.68181e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 6.35379e+11\tvalid_0's mse_score: 6.35379e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.72742e+11\tvalid_0's mse_score: 3.72742e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 7.63386e+11\tvalid_0's mse_score: 7.63386e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 4.07924e+11\tvalid_0's mse_score: 4.07924e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 7.44934e+11\tvalid_0's mse_score: 7.44934e+11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 2/22 [01:04<11:07, 33.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.1371e+10\tvalid_0's mse_score: 6.1371e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 8.07529e+10\tvalid_0's mse_score: 8.07529e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 5.63397e+10\tvalid_0's mse_score: 5.63397e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 7.25488e+10\tvalid_0's mse_score: 7.25488e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 5.13772e+10\tvalid_0's mse_score: 5.13772e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 6.75974e+10\tvalid_0's mse_score: 6.75974e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.1303e+10\tvalid_0's mse_score: 3.1303e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.0604e+10\tvalid_0's mse_score: 4.0604e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 8.72866e+10\tvalid_0's mse_score: 8.72866e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.14079e+11\tvalid_0's mse_score: 1.14079e+11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▎        | 3/22 [12:04<1:10:08, 221.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 5.57467e+12\tvalid_0's mse_score: 5.57467e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 5.57467e+12\tvalid_0's mse_score: 5.57467e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 4.8294e+12\tvalid_0's mse_score: 4.8294e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.8294e+12\tvalid_0's mse_score: 4.8294e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.79117e+12\tvalid_0's mse_score: 3.79117e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 3.79117e+12\tvalid_0's mse_score: 3.79117e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 4.67316e+12\tvalid_0's mse_score: 4.67316e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.67316e+12\tvalid_0's mse_score: 4.67316e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 5.24864e+12\tvalid_0's mse_score: 5.24864e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 5.24864e+12\tvalid_0's mse_score: 5.24864e+12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 4/22 [12:07<46:46, 155.90s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 4.83357e+12\tvalid_0's mse_score: 4.83357e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.83357e+12\tvalid_0's mse_score: 4.83357e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 9.12484e+12\tvalid_0's mse_score: 9.12484e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 9.12484e+12\tvalid_0's mse_score: 9.12484e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.01967e+13\tvalid_0's mse_score: 1.01967e+13\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.01967e+13\tvalid_0's mse_score: 1.01967e+13\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 5.72719e+12\tvalid_0's mse_score: 5.72719e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 5.72719e+12\tvalid_0's mse_score: 5.72719e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.46953e+12\tvalid_0's mse_score: 3.46953e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 3.46953e+12\tvalid_0's mse_score: 3.46953e+12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 5/22 [12:10<31:12, 110.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.13261e+11\tvalid_0's mse_score: 1.13261e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.44653e+11\tvalid_0's mse_score: 2.44653e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 7.08065e+10\tvalid_0's mse_score: 7.08065e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.75264e+11\tvalid_0's mse_score: 1.75264e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 8.7102e+10\tvalid_0's mse_score: 8.7102e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.05989e+11\tvalid_0's mse_score: 2.05989e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.3848e+11\tvalid_0's mse_score: 1.3848e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.36357e+11\tvalid_0's mse_score: 2.36357e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 7.84793e+10\tvalid_0's mse_score: 7.84793e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.0973e+11\tvalid_0's mse_score: 2.0973e+11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 6/22 [12:51<23:52, 89.51s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 5.11763e+12\tvalid_0's mse_score: 5.11763e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 5.11763e+12\tvalid_0's mse_score: 5.11763e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 2.58971e+12\tvalid_0's mse_score: 2.58971e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.58971e+12\tvalid_0's mse_score: 2.58971e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 2.58432e+12\tvalid_0's mse_score: 2.58432e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.58432e+12\tvalid_0's mse_score: 2.58432e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 2.09548e+12\tvalid_0's mse_score: 2.09548e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.09548e+12\tvalid_0's mse_score: 2.09548e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 4.27724e+12\tvalid_0's mse_score: 4.27724e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.27724e+12\tvalid_0's mse_score: 4.27724e+12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 7/22 [12:57<16:02, 64.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.17092e+12\tvalid_0's mse_score: 1.17092e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.17092e+12\tvalid_0's mse_score: 1.17092e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 9.18362e+11\tvalid_0's mse_score: 9.18362e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 9.18362e+11\tvalid_0's mse_score: 9.18362e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.49309e+12\tvalid_0's mse_score: 1.49309e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.49309e+12\tvalid_0's mse_score: 1.49309e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 9.727e+11\tvalid_0's mse_score: 9.727e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 9.727e+11\tvalid_0's mse_score: 9.727e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.06007e+12\tvalid_0's mse_score: 1.06007e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.06007e+12\tvalid_0's mse_score: 1.06007e+12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▋      | 8/22 [12:59<10:37, 45.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.39914e+13\tvalid_0's mse_score: 1.39914e+13\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.39914e+13\tvalid_0's mse_score: 1.39914e+13\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.39914e+13\tvalid_0's mse_score: 1.39914e+13\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.39914e+13\tvalid_0's mse_score: 1.39914e+13\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.39914e+13\tvalid_0's mse_score: 1.39914e+13\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.39914e+13\tvalid_0's mse_score: 1.39914e+13\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.39914e+13\tvalid_0's mse_score: 1.39914e+13\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.39914e+13\tvalid_0's mse_score: 1.39914e+13\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.39914e+13\tvalid_0's mse_score: 1.39914e+13\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.39914e+13\tvalid_0's mse_score: 1.39914e+13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 9/22 [12:59<06:55, 31.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.28204e+11\tvalid_0's mse_score: 3.28204e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.13908e+11\tvalid_0's mse_score: 4.13908e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.34994e+11\tvalid_0's mse_score: 1.34994e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.07841e+11\tvalid_0's mse_score: 2.07841e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.63794e+11\tvalid_0's mse_score: 1.63794e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.18361e+11\tvalid_0's mse_score: 2.18361e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.83132e+11\tvalid_0's mse_score: 1.83132e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.07626e+11\tvalid_0's mse_score: 2.07626e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.1241e+11\tvalid_0's mse_score: 1.1241e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.81909e+11\tvalid_0's mse_score: 1.81909e+11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 10/22 [13:53<07:44, 38.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.0729e+11\tvalid_0's mse_score: 1.0729e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.97261e+11\tvalid_0's mse_score: 1.97261e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.62378e+10\tvalid_0's mse_score: 6.62378e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.27122e+11\tvalid_0's mse_score: 1.27122e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 8.17383e+10\tvalid_0's mse_score: 8.17383e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.33912e+11\tvalid_0's mse_score: 1.33912e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.03613e+11\tvalid_0's mse_score: 1.03613e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.76877e+11\tvalid_0's mse_score: 1.76877e+11\n",
      "Training until validation scores don't improve for 100 rounds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 11/22 [14:40<07:32, 41.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100]\tvalid_0's l2: 1.7249e+11\tvalid_0's mse_score: 1.7249e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.46518e+11\tvalid_0's mse_score: 2.46518e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.40825e+12\tvalid_0's mse_score: 1.40825e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.40825e+12\tvalid_0's mse_score: 1.40825e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.16746e+12\tvalid_0's mse_score: 1.16746e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.16746e+12\tvalid_0's mse_score: 1.16746e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.45488e+12\tvalid_0's mse_score: 1.45488e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.45488e+12\tvalid_0's mse_score: 1.45488e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.61215e+12\tvalid_0's mse_score: 1.61215e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.61215e+12\tvalid_0's mse_score: 1.61215e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.12495e+12\tvalid_0's mse_score: 1.12495e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.12495e+12\tvalid_0's mse_score: 1.12495e+12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▍    | 12/22 [14:46<05:06, 30.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.63887e+12\tvalid_0's mse_score: 6.63887e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 6.63887e+12\tvalid_0's mse_score: 6.63887e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.51543e+12\tvalid_0's mse_score: 6.51543e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 6.51543e+12\tvalid_0's mse_score: 6.51543e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.16655e+12\tvalid_0's mse_score: 6.16655e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 6.16655e+12\tvalid_0's mse_score: 6.16655e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.66399e+12\tvalid_0's mse_score: 6.66399e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 6.66399e+12\tvalid_0's mse_score: 6.66399e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.65448e+12\tvalid_0's mse_score: 6.65448e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 6.65448e+12\tvalid_0's mse_score: 6.65448e+12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 13/22 [14:47<03:14, 21.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 2.46665e+11\tvalid_0's mse_score: 2.46665e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.20479e+11\tvalid_0's mse_score: 4.20479e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.90742e+11\tvalid_0's mse_score: 1.90742e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 3.24851e+11\tvalid_0's mse_score: 3.24851e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.13873e+11\tvalid_0's mse_score: 3.13873e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 5.21259e+11\tvalid_0's mse_score: 5.21259e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.45736e+11\tvalid_0's mse_score: 3.45736e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 5.21573e+11\tvalid_0's mse_score: 5.21573e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.8212e+11\tvalid_0's mse_score: 3.8212e+11\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 5.36436e+11\tvalid_0's mse_score: 5.36436e+11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▎   | 14/22 [15:11<03:00, 22.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.49684e+10\tvalid_0's mse_score: 6.49684e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.48867e+11\tvalid_0's mse_score: 1.48867e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.8708e+10\tvalid_0's mse_score: 6.8708e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.53913e+11\tvalid_0's mse_score: 1.53913e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 7.45259e+10\tvalid_0's mse_score: 7.45259e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.55936e+11\tvalid_0's mse_score: 1.55936e+11\n",
      "Training until validation scores don't improve for 100 rounds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 15/22 [17:28<06:37, 56.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100]\tvalid_0's l2: 7.72891e+10\tvalid_0's mse_score: 7.72891e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 1.63267e+11\tvalid_0's mse_score: 1.63267e+11\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.58875e+10\tvalid_0's mse_score: 1.58875e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 6.24912e+10\tvalid_0's mse_score: 6.24912e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 2.88211e+12\tvalid_0's mse_score: 2.88211e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.88211e+12\tvalid_0's mse_score: 2.88211e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 2.89109e+12\tvalid_0's mse_score: 2.89109e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.89109e+12\tvalid_0's mse_score: 2.89109e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.38169e+12\tvalid_0's mse_score: 3.38169e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 3.38169e+12\tvalid_0's mse_score: 3.38169e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 4.0166e+12\tvalid_0's mse_score: 4.0166e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.0166e+12\tvalid_0's mse_score: 4.0166e+12\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 2.54702e+12\tvalid_0's mse_score: 2.54702e+12\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.54702e+12\tvalid_0's mse_score: 2.54702e+12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 16/22 [17:35<04:10, 41.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.82831e+09\tvalid_0's mse_score: 6.82831e+09\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.91334e+10\tvalid_0's mse_score: 2.91334e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 5.74217e+09\tvalid_0's mse_score: 5.74217e+09\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.67418e+10\tvalid_0's mse_score: 2.67418e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 7.93426e+09\tvalid_0's mse_score: 7.93426e+09\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 3.24161e+10\tvalid_0's mse_score: 3.24161e+10\n",
      "Training until validation scores don't improve for 100 rounds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 17/22 [17:36<02:28, 29.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100]\tvalid_0's l2: 1.75521e+10\tvalid_0's mse_score: 1.75521e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 5.17963e+10\tvalid_0's mse_score: 5.17963e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 7.13764e+09\tvalid_0's mse_score: 7.13764e+09\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.90534e+10\tvalid_0's mse_score: 2.90534e+10\n"
     ]
    }
   ],
   "source": [
    "for route in tqdm(test_order_belong_to_trace):\n",
    "    route_order_info = train_order_data[route]\n",
    "    train_data = get_train_data(route_order_info, route)\n",
    "    features = [c for c in train_data.columns if c not in ['loadingOrder', 'label']]\n",
    "    model_by_route = train_model(train_data[features], train_data['label'])\n",
    "    \n",
    "    for order in test_order_belong_to_trace[route]:\n",
    "        test_order_data = get_test_data(order)\n",
    "        res = model_by_route.predict(test_order_data[features], num_iteration=model_by_route.best_iteration)\n",
    "        test_data_origin.loc[test_data_origin['loadingOrder'] == order, 'ETA'] = (test_data_origin[test_data_origin['loadingOrder'] == order]['onboardDate'] + pd.Timedelta(seconds=res[0])).apply(lambda x:x.strftime('%Y/%m/%d  %H:%M:%S'))\n",
    "    \n",
    "test_data_origin['creatDate'] = pd.datetime.now().strftime('%Y/%m/%d  %H:%M:%S')\n",
    "test_data_origin['timestamp'] = test_data_origin['temp_timestamp']\n",
    "\n",
    "result = test_data_origin[['loadingOrder', 'timestamp', 'longitude', 'latitude', 'carrierName', 'vesselMMSI', 'onboardDate', 'ETA', 'creatDate']]\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_csv(result_path, index=False)\n",
    "mox.file.copy_parallel(result_path, OBS_RES_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit ('AI': conda)",
   "language": "python",
   "name": "python37664bitaiconda6859e03b37c34f0182c9bde8073269f7"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
